'''
Created on 2011-10-26

@author: Administrator
'''
import os
import urllib2
import sqlite3
import datetime
import threading
import Queue
import time
import shutil 
from setting import *
    

class ThreadStockDataDownload(threading.Thread):
    def __init__(self):
        threading.Thread.__init__(self)
        self.queue=urlqueue
    def run(self):
        job=self.queue.get()  
        p=job.find('.')
        reg=job[:2]
        if reg in ['SH','SZ']:reg='CH'
        if reg in ['NS','NQ','AS']:reg='US'
        stockcode=job[2:p]+':'+reg
        if reg=='HK':stockcode=stockcode.strip('0')
#        print 'thread stockcode:',stockcode
        url='http://www.bloomberg.com/apps/data?pid=webpxta&Securities='+stockcode+'&TimePeriod=5Y&Outfields=HDATE,PR006-H,PR007-H,PR008-H,PR005-H,PR013-H'
        
        stockcode2=job.replace(':','_')
        try:
            content=urllib2.urlopen(url).read()
        except:
#            print '[x]',#url   
            fname=os.path.join(errordatapath,job)
            f=open(fname,'w')
#            f.write(content)
            f.close()        
            self.queue.task_done()  
            return
        if content[:5]=='Error':
#            print '[r]',#url
            fname=os.path.join(errordatapath,job)
            f=open(fname,'w')
#            f.write(content)
            f.close() 
            self.queue.task_done()
            return
        else:
            clist=content.split('\n')
            tlist=''
            for t in range(1,len(clist)-2):
                tlist=tlist+clist[t]+'\n'
            nlist=job.split(':')
            fname=os.path.join(rawcontentpath,job)
            f=open(fname,'w')
            f.write(tlist)
#            f.write(tlist.replace('"',' '))
            f.close()  
            filetobedelete=os.path.join(jobtodopath,job)
            if os.path.exists(filetobedelete):
                try:
                    os.remove(filetobedelete)
                except:
                    print 'remove error',filetobedelete
            self.queue.task_done()
            return
        self.__stop()
class ThreadMain(threading.Thread):
    def __init__(self,num):
        threading.Thread.__init__(self)
        self.num=num
    def run(self):          
#        print 'g_jobsnumber',self.num
        for i in range(self.num):
            dt=ThreadStockDataDownload()
            dt.start()
        urlqueue.join()

class StockDataDownload():
    def __init__(self):
        if not os.path.exists(rawcontentpath):os.mkdir(rawcontentpath)
        if not os.path.exists(errordatapath):os.mkdir(errordatapath)
        if not os.path.exists(jobtodopath):os.mkdir(jobtodopath)
        if not os.path.exists(stockbycodepath):os.mkdir(stockbycodepath)
        if not os.path.exists(stockbydatepath):os.mkdir(stockbydatepath)
        for reg in reglist:
            dbpath=os.path.join(stockbycodepath,reg)
            if not os.path.exists(dbpath):os.mkdir(dbpath)
#            self.clearpathfile(dbpath)
        for reg in reglist:
            bydatepath=os.path.join(stockbydatepath,reg)
            if not os.path.exists(bydatepath):os.mkdir(bydatepath)
    def clearpathfile(self,mypath):
        for root,dirs,files in os.walk(mypath):
            for f in files:
                os.remove(os.path.join(root,f)) 
    def movepathfile(self,srcpath,dstpath):
        for root,dirs,files in os.walk(srcpath):
            for f in files:
                dstfile=os.path.join(dstpath,f)
                fdst=open(dstfile,'w')
                fdst.close()
                os.remove(os.path.join(root,f))          
    def geterrorjobslist(self):
        errorlist=self.getpathfilelist(errordatapath)
        return errorlist
    def getpathfilelist(self,mypath,filter=''):
        flist=[]
        for root,dirs,files in os.walk(mypath):
            for f in files:
                if filter=='':flist.append(f.replace('_',':'))
                else:
                    if f.find(filter)>0:flist.append(f.replace('_',':'))
        return flist     
   
    def downloadstart(self,stockall):
        def downloadround():
            todolist=[]
            todolist=self.getsomejobs()
            g_jobsnumber=len(todolist) 
    #        print 'this round todo:',len(todolist)
            while g_jobsnumber>0:
#                print 'this round todo:',g_jobsnumber,datetime.datetime.now()
                for job in todolist:
                    urlqueue.put(job,block=True ,timeout= threadtimeout)             
                td=ThreadMain(g_jobsnumber)
                td.start()
                td.join(manthreadtimeout)
                todolist=self.getsomejobs()
                g_jobsnumber=len(todolist)   
            errorlist=self.geterrorjobslist()
            return len(errorlist) 
        ############################################           
        self.clearpathfile(rawcontentpath)
        self.clearpathfile(errordatapath)
        self.clearpathfile(jobtodopath)
#        jobs=StockList()
        ######################################
#        print stockall
        jobnum=0
        for reg in stockall:
            for stock in stockall[reg].keys():                
                fname=os.path.join(jobtodopath,reg+stock+'.txt')
                f=open(fname,'w')
                f.close()   
                jobnum=jobnum+1
##################################################
        i=0
        print 'round start',i
        self.movepathfile(errordatapath,jobtodopath)
        errornum=downloadround()
        print '##########',i,jobnum,errornum
        while errornum!=jobnum:
            jobnum=errornum
            i=i+1
            print 'round start',i
            self.movepathfile(errordatapath,jobtodopath)
            errornum=downloadround()
            print '##########',i,jobnum,errornum
##############################################

    def getsomejobs(self):
        todolist=[]
        joblist=self.getpathfilelist(jobtodopath)
        errorlist=self.geterrorjobslist()
        for job in joblist:
            if job not in errorlist:
                todolist.append(job)
#        print 'left job:',len(todolist)
        print len(todolist),
        if todolist==[]:return todolist
        else:            
            jobnum=len(todolist)
#            print 'go here',jobnum,workers_number,jobnum<=workers_number
            if jobnum<=workers_number:
                return todolist
            else:
#                print todolist[:workers_number]
                return todolist[:workers_number]

    def checkstockfile(self):
        rawlist=[]
        for root,dirs,files in os.walk(rawcontentpath):
            for f in files:
                rawlist.append(f)
                fname=os.path.join(root,f)
                f=open(fname,'r')
                content=f.read()
                f.close()
                tlist=content.split('\n')
                s=tlist[0].find('Count')
                p=tlist[0].find('=',s)
                q=tlist[0].find('"',p)
                num=int(tlist[0][p+1:q])
                flines=len(tlist)
                if flines==num+3:pass
                else:print 'checkstockfile failed:',len(tlist),num,fname
                
        print 'raw file num:',len(rawlist),rawlist[0]
        dblist=[]
        for reg in reglist:
            rlist=self.getpathfilelist(os.path.join(stockbycodepath,reg))
            for r in rlist:
                p=r.find('.')
                dblist.append(r[:p]+'_'+reg)
        print 'db file num:',len(dblist),dblist[0]
        
        for raw in rawlist:
            if raw in dblist:pass
            else:print raw,'not in dblist'
        return


if __name__ == '__main__':
    starttime=datetime.datetime.now()
    print starttime   


    print 'download OK',starttime,(datetime.datetime.now()-starttime).seconds
#    sj.dostockdb()
#    print 'dostockdb OK',starttime,datetime.datetime.now()
#    sj.checkstockfile()
#    print 'checkstockfile OK',starttime,datetime.datetime.now()
#    sj.trans2bydate()
#    print 'start and end time:',starttime,datetime.datetime.now()
#    clearallfile()